<!DOCTYPE html>
<html lang="zh-CN">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>PPM 训练数据构建过程可视化</title>
    <script src="https://cdn.jsdelivr.net/npm/mermaid@10.6.1/dist/mermaid.min.js"></script>
    <style>
        body {
            font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
            margin: 0;
            padding: 20px;
            background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
            min-height: 100vh;
            line-height: 1.6;
        }
        .container {
            max-width: 1600px;
            margin: 0 auto;
            background: white;
            border-radius: 15px;
            box-shadow: 0 10px 30px rgba(0, 0, 0, 0.2);
            overflow: hidden;
        }
        .header {
            background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
            color: white;
            padding: 40px;
            text-align: center;
            position: relative;
        }
        .header::before {
            content: '';
            position: absolute;
            top: 0;
            left: 0;
            right: 0;
            bottom: 0;
            background: url('data:image/svg+xml,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 100"><defs><pattern id="grain" width="100" height="100" patternUnits="userSpaceOnUse"><circle cx="50" cy="50" r="1" fill="%23ffffff" opacity="0.1"/></pattern></defs><rect width="100" height="100" fill="url(%23grain)"/></svg>') repeat;
        }
        .header h1 {
            margin: 0;
            font-size: 3em;
            font-weight: 300;
            position: relative;
            z-index: 1;
        }
        .header p {
            margin: 15px 0 0 0;
            font-size: 1.3em;
            opacity: 0.9;
            position: relative;
            z-index: 1;
        }
        .content {
            padding: 40px;
        }
        .section {
            margin-bottom: 50px;
        }
        .section h2 {
            color: #2c3e50;
            border-bottom: 4px solid #3498db;
            padding-bottom: 15px;
            margin-bottom: 30px;
            font-size: 2em;
            font-weight: 400;
        }
        .visualization-container {
            display: flex;
            gap: 30px;
            margin: 30px 0;
        }
        .tree-section {
            flex: 1;
            background: #f8f9fa;
            border-radius: 12px;
            padding: 25px;
            box-shadow: 0 4px 15px rgba(0, 0, 0, 0.1);
        }
        .samples-section {
            flex: 1;
            background: #f8f9fa;
            border-radius: 12px;
            padding: 25px;
            box-shadow: 0 4px 15px rgba(0, 0, 0, 0.1);
        }
        .mermaid {
            text-align: center;
            margin: 20px 0;
            background: white;
            border-radius: 8px;
            padding: 20px;
            box-shadow: 0 2px 10px rgba(0, 0, 0, 0.1);
        }
        .layer-samples {
            margin: 20px 0;
        }
        .layer-title {
            font-weight: bold;
            color: #2c3e50;
            margin-bottom: 15px;
            font-size: 1.2em;
            padding: 10px;
            background: linear-gradient(135deg, #3498db, #2980b9);
            color: white;
            border-radius: 8px;
            text-align: center;
        }
        .sample-pair {
            display: flex;
            gap: 15px;
            margin: 15px 0;
            padding: 15px;
            background: white;
            border-radius: 8px;
            box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1);
        }
        .positive-sample, .negative-sample {
            flex: 1;
            padding: 15px;
            border-radius: 8px;
            font-family: 'Courier New', monospace;
            font-size: 0.9em;
            line-height: 1.4;
        }
        .positive-sample {
            background: linear-gradient(135deg, #d4edda, #c3e6cb);
            border-left: 4px solid #28a745;
        }
        .negative-sample {
            background: linear-gradient(135deg, #f8d7da, #f5c6cb);
            border-left: 4px solid #dc3545;
        }
        .sample-header {
            font-weight: bold;
            margin-bottom: 8px;
            font-family: 'Segoe UI', sans-serif;
        }
        .positive-sample .sample-header {
            color: #155724;
        }
        .negative-sample .sample-header {
            color: #721c24;
        }
        .q-value {
            font-size: 0.8em;
            opacity: 0.8;
            margin-top: 8px;
            font-style: italic;
        }
        .description {
            background: linear-gradient(135deg, #e3f2fd, #bbdefb);
            padding: 25px;
            border-radius: 12px;
            border-left: 5px solid #2196f3;
            margin: 25px 0;
            box-shadow: 0 4px 15px rgba(0, 0, 0, 0.1);
        }
        .phase-box {
            background: linear-gradient(135deg, #f3e5f5, #e1bee7);
            border: 1px solid #ce93d8;
            border-radius: 12px;
            padding: 20px;
            margin: 20px 0;
            box-shadow: 0 4px 15px rgba(0, 0, 0, 0.1);
        }
        .phase-title {
            font-weight: bold;
            color: #4a148c;
            margin-bottom: 15px;
            font-size: 1.3em;
        }
        .code-ref {
            font-family: 'Courier New', monospace;
            background: linear-gradient(135deg, #f1f2f6, #e8eaf6);
            padding: 4px 8px;
            border-radius: 4px;
            font-size: 0.9em;
            border: 1px solid #c5cae9;
        }
        .legend {
            background: linear-gradient(135deg, #fff3e0, #ffe0b2);
            border: 1px solid #ffcc02;
            border-radius: 12px;
            padding: 25px;
            margin: 25px 0;
            box-shadow: 0 4px 15px rgba(0, 0, 0, 0.1);
        }
        .legend h3 {
            margin-top: 0;
            color: #e65100;
            font-size: 1.4em;
        }
        .legend-item {
            display: flex;
            align-items: center;
            margin: 12px 0;
            font-weight: 500;
        }
        .legend-color {
            width: 24px;
            height: 24px;
            border-radius: 6px;
            margin-right: 12px;
            box-shadow: 0 2px 4px rgba(0, 0, 0, 0.2);
        }
        .correct-color { background: linear-gradient(135deg, #4caf50, #388e3c); }
        .wrong-color { background: linear-gradient(135deg, #f44336, #d32f2f); }
        .processing-color { background: linear-gradient(135deg, #2196f3, #1976d2); }
        .selected-color { background: linear-gradient(135deg, #ff9800, #f57c00); }
        .comparison-color { background: linear-gradient(135deg, #9c27b0, #7b1fa2); }
        
        /* Mermaid 样式优化 */
        .mermaid .node rect {
            stroke-width: 2px;
        }
        .mermaid .node text {
            font-family: 'Segoe UI', sans-serif;
            font-size: 12px;
            font-weight: 500;
        }
        .mermaid .edgePath {
            stroke-width: 2px;
        }
        
        /* 响应式设计 */
        @media (max-width: 1200px) {
            .visualization-container {
                flex-direction: column;
            }
        }
        
        /* 动画效果 */
        .sample-pair {
            transition: transform 0.3s ease, box-shadow 0.3s ease;
        }
        .sample-pair:hover {
            transform: translateY(-2px);
            box-shadow: 0 4px 20px rgba(0, 0, 0, 0.15);
        }
        
        .phase-box {
            transition: transform 0.3s ease;
        }
        .phase-box:hover {
            transform: translateY(-1px);
        }
    </style>
</head>
<body>
    <div class="container">
        <div class="header">
            <h1>PPM 训练数据构建过程可视化</h1>
            <p>过程偏好模型 (Process Preference Model) 的数据生成与训练流程</p>
        </div>
        
        <div class="content">
            <div class="section">
                <h2>🎯 PPM 核心思想</h2>
                <div class="description">
                    <p><strong>PPM (Process Preference Model)</strong> 是 rStar-Math 框架的核心创新，它通过学习<strong>相对偏好关系</strong>而非绝对分数来评估推理步骤的质量。</p>
                    <p><strong>核心优势：</strong>相对比较比绝对评分更可靠，能够有效处理 MCTS Q-value 中的噪声问题。</p>
                </div>
                
                <div class="phase-box">
                    <div class="phase-title">📊 PPM 训练数据规模与特点</div>
                    <ul>
                        <li><strong>样本对数量：</strong>每棵MCTS树产生的正负样本对数量<strong>不固定</strong>，取决于树的深度、广度和终端节点分布</li>
                        <li><strong>动态平衡：</strong>通过 <span class="code-ref">min(len(pos_nodes), len(neg_nodes))</span> 确保正负样本数量平衡</li>
                        <li><strong>典型规模：</strong>一棵MCTS树通常产生几十到几百个正负样本对</li>
                        <li><strong>质量控制：</strong>只保留 <span class="code-ref">step_margin > 0</span> 且 <span class="code-ref">steps_margin > 0</span> 的高质量偏好对</li>
                    </ul>
                </div>
                
                <div class="phase-box">
                    <div class="phase-title">🛤️ 轨迹构建策略</div>
                    <p><strong>关键特点：</strong>每个样本对<strong>并非完整的根节点到终端节点路径</strong>，而是采用精细的轨迹拼接策略：</p>
                    <ul>
                        <li><strong>前缀路径：</strong>从根节点到当前节点的完整推理历史</li>
                        <li><strong>当前步骤：</strong>当前节点的具体推理操作</li>
                        <li><strong>后续路径：</strong>从当前节点到最优/最差终端节点的路径</li>
                        <li><strong>层次化训练：</strong>不仅训练终端节点，也训练中间决策点</li>
                        <li><strong>对比学习：</strong>同一前缀下的不同后续路径形成有效对比</li>
                    </ul>
                </div>
                
                <div class="phase-box">
                    <div class="phase-title">🎯 PPM 中间节点评分机制</div>
                    <p><strong>评分流程：</strong>PPM在MCTS推理过程中对中间节点进行实时评分，指导搜索方向：</p>
                    <ul>
                        <li><strong>输入构建：</strong><span class="code-ref">value_preprocess</span> 为每个节点构建评估提示词</li>
                        <li><strong>价值估计：</strong>调用 <span class="code-ref">reward_model</span> 对从根节点到当前节点的完整路径进行评分</li>
                        <li><strong>节点更新：</strong><span class="code-ref">value_postprocess</span> 处理模型输出，更新节点的 <span class="code-ref">value_estimate</span></li>
                        <li><strong>选择策略：</strong><span class="code-ref">select_next_step</span> 使用UCB公式结合价值估计选择最优子节点</li>
                        <li><strong>一致性保证：</strong>评估的完整路径格式与训练数据保持一致，确保模型泛化能力</li>
                    </ul>
                    <div class="code-block">
                        <strong>核心评分公式：</strong><br>
                        UCB_score = Q_value + C * sqrt(log(parent_visits) / node_visits) + value_estimate
                    </div>
                </div>
            </div>

            <div class="section">
                <h2>🔄 MCTS 搜索树与 PPM 数据构建</h2>
                <div class="legend">
                    <h3>图例说明</h3>
                    <div class="legend-item">
                        <div class="legend-color correct-color"></div>
                        <span>正确路径 (导向正确答案)</span>
                    </div>
                    <div class="legend-item">
                        <div class="legend-color wrong-color"></div>
                        <span>错误路径 (导向错误答案)</span>
                    </div>
                    <div class="legend-item">
                        <div class="legend-color processing-color"></div>
                        <span>中间节点 (推理过程中)</span>
                    </div>
                    <div class="legend-item">
                        <div class="legend-color selected-color"></div>
                        <span>被选中节点 (用于构建训练数据)</span>
                    </div>
                    <div class="legend-item">
                        <div class="legend-color comparison-color"></div>
                        <span>偏好对比较</span>
                    </div>
                </div>
                
                <div class="visualization-container">
                    <div class="tree-section">
                        <h3 style="text-align: center; color: #2c3e50; margin-bottom: 20px;">🌳 MCTS 搜索树结构</h3>
                        <div class="mermaid">
                            flowchart TD
                                Root(["问题: 求解 2(x+3)=14"]) 
                                
                                %% 第一层
                                Root --> A1["步骤1.1: 两边除以2<br/>Q=0.95 ✓"]
                                Root --> A2["步骤1.2: 移项变换<br/>Q=0.80 ✓"]
                                Root --> A3["步骤1.3: 错误计算<br/>Q=-0.60 ✗"]
                                Root --> A4["步骤1.4: 错误操作<br/>Q=-0.90 ✗"]
                                
                                %% 第二层 - 正确路径
                                A1 --> B1["步骤2.1: x+3=7<br/>减去3<br/>Q=0.92 ✓"]
                                A1 --> B2["步骤2.2: x+3=7<br/>验证步骤<br/>Q=0.85 ✓"]
                                A2 --> B3["步骤2.3: 3-7=-x<br/>化简<br/>Q=0.75 ✓"]
                                
                                %% 第二层 - 错误路径
                                A3 --> B4["步骤2.4: 计算错误<br/>x=5<br/>Q=-0.80 ✗"]
                                A4 --> B5["步骤2.5: 逻辑错误<br/>x=10<br/>Q=-0.95 ✗"]
                                
                                %% 第三层 - 最终答案
                                B1 --> C1["最终答案: x=4<br/>✓ 正确"]
                                B2 --> C2["最终答案: x=4<br/>✓ 正确"]
                                B3 --> C3["最终答案: x=4<br/>✓ 正确"]
                                B4 --> C4["最终答案: x=5<br/>✗ 错误"]
                                B5 --> C5["最终答案: x=10<br/>✗ 错误"]
                                
                                %% 样式定义
                                classDef correctNode fill:#c8e6c9,stroke:#4caf50,stroke-width:3px,color:#1b5e20
                                classDef wrongNode fill:#ffcdd2,stroke:#f44336,stroke-width:3px,color:#b71c1c
                                classDef selectedPos fill:#fff3e0,stroke:#ff9800,stroke-width:4px,color:#e65100
                                classDef selectedNeg fill:#fce4ec,stroke:#e91e63,stroke-width:4px,color:#880e4f
                                classDef rootNode fill:#e3f2fd,stroke:#2196f3,stroke-width:3px,color:#0d47a1
                                
                                class Root rootNode
                                class A1,A2,B1,B2,B3,C1,C2,C3 correctNode
                                class A3,A4,B4,B5,C4,C5 wrongNode
                                class A1,B1 selectedPos
                                class A4,B5 selectedNeg
                        </div>
                    </div>
                    
                    <div class="samples-section">
                        <h3 style="text-align: center; color: #2c3e50; margin-bottom: 20px;">📊 每层构建的偏好对</h3>
                        
                        <div class="layer-samples">
                            <div class="layer-title">第一层: 初始步骤选择</div>
                            <div class="sample-pair">
                                <div class="positive-sample">
                                    <div class="sample-header">✅ 正样本 (Chosen)</div>
                                    <div># Step 1: 两边同时除以2<br/>x + 3 = 7</div>
                                    <div class="q-value">Q-value: 0.95 | 导向正确答案</div>
                                </div>
                                <div class="negative-sample">
                                    <div class="sample-header">❌ 负样本 (Rejected)</div>
                                    <div># Step 1: 两边同时加3<br/>2(x+3) + 3 = 14 + 3</div>
                                    <div class="q-value">Q-value: -0.90 | 导向错误答案</div>
                                </div>
                            </div>
                        </div>
                        
                        <div class="layer-samples">
                            <div class="layer-title">第二层: 后续推理步骤 (展示轨迹拼接)</div>
                            <div class="sample-pair">
                                <div class="positive-sample">
                                    <div class="sample-header">✅ 正样本 (Chosen) - 完整轨迹构建</div>
                                    <div><strong>前缀:</strong> &lt;|user|&gt;: 求解 2(x+3)=14<br/>&lt;|assistant|&gt;: Let's think step by step...<br/># Step 1: 两边除以2<br/>x + 3 = 7<br/><br/><strong>当前节点:</strong> # Step 2: 两边减去3<br/>x = 7 - 3<br/><br/><strong>后续最优路径:</strong> x = 4<br/>验证: 2(4+3) = 14 ✓</div>
                                    <div class="q-value">当前节点 Q-value: 0.92 | 后续路径平均分: 0.95</div>
                                </div>
                                <div class="negative-sample">
                                    <div class="sample-header">❌ 负样本 (Rejected) - 完整轨迹构建</div>
                                    <div><strong>前缀:</strong> &lt;|user|&gt;: 求解 2(x+3)=14<br/>&lt;|assistant|&gt;: Let's think step by step...<br/># Step 1: 两边除以2<br/>x + 3 = 7<br/><br/><strong>当前节点:</strong> # Step 2: 两边加上3<br/>x = 7 + 3<br/><br/><strong>后续最差路径:</strong> x = 10<br/>验证: 2(10+3) = 26 ≠ 14 ✗</div>
                                    <div class="q-value">当前节点 Q-value: -0.95 | 后续路径平均分: -0.90</div>
                                </div>
                            </div>
                        </div>
                        
                        <div class="layer-samples">
                            <div class="layer-title">第三层: 最终答案验证 (终端节点处理)</div>
                            <div class="sample-pair">
                                <div class="positive-sample">
                                    <div class="sample-header">✅ 正样本 (Chosen) - 终端节点</div>
                                    <div><strong>完整轨迹:</strong><br/>前缀 + 当前路径 + 空后续路径<br/><br/><strong>最终形式:</strong><br/>&lt;|user|&gt;: 求解 2(x+3)=14<br/>&lt;|assistant|&gt;: Let's think step by step...<br/># Step 1: 两边除以2<br/>x + 3 = 7<br/># Step 2: 两边减去3<br/>x = 7 - 3<br/>x = 4<br/>验证: 2(4+3) = 14 ✓</div>
                                    <div class="q-value">终端节点 Q-value: +1.0 (is_valid_final_answer_node = True)</div>
                                </div>
                                <div class="negative-sample">
                                    <div class="sample-header">❌ 负样本 (Rejected) - 终端节点</div>
                                    <div><strong>完整轨迹:</strong><br/>前缀 + 当前路径 + 空后续路径<br/><br/><strong>最终形式:</strong><br/>&lt;|user|&gt;: 求解 2(x+3)=14<br/>&lt;|assistant|&gt;: Let's think step by step...<br/># Step 1: 错误操作<br/>2(x+3) + 3 = 17<br/># Step 2: 继续错误<br/>x = 10<br/>验证: 2(10+3) = 26 ≠ 14 ✗</div>
                                    <div class="q-value">终端节点 Q-value: -1.0 (答案错误)</div>
                                </div>
                            </div>
                        </div>
                    </div>
                </div>
            </div>

            <div class="section">
                <h2>📋 PPM 训练数据构建流程</h2>
                
                <div class="phase-box">
                    <div class="phase-title">🔍 第一步: MCTS 搜索与 Q-value 计算</div>
                    <p><strong>目标：</strong>通过多轮 rollout 探索不同推理路径，计算每个节点的 Q-value</p>
                    <ul>
                        <li><span class="code-ref">search_all_traces(root_node)</span>: 深度优先遍历 MCTS 树的入口函数</li>
                        <li><span class="code-ref">dfs(node, depth=0)</span>: 递归遍历，标记终端节点状态</li>
                        <li><span class="code-ref">node.q_value</span>: 基于 MCTS 回传的价值估计 (+1 正确, -1 错误)</li>
                        <li><span class="code-ref">node.final_correct/final_wrong</span>: 统计导向正确/错误答案的路径数</li>
                        <li><strong>终端判断：</strong><span class="code-ref">is_valid_final_answer_node(node)</span> 检查是否包含最终答案</li>
                        <li><strong>状态回传：</strong>子节点的 final_correct/final_wrong 累加到父节点</li>
                        <li><strong>层次遍历：</strong>按深度分组，为每层生成训练数据做准备</li>
                    </ul>
                </div>

                <div class="phase-box">
                    <div class="phase-title">🎯 第二步: 正负样本筛选</div>
                    <p><strong>目标：</strong>在每一层选择最优的正样本和最差的负样本</p>
                    <ul>
                        <li><strong>正样本候选：</strong><span class="code-ref">pos_nodes = [child for child in nodes if child.final_correct > 0]</span></li>
                        <li><strong>负样本候选：</strong><span class="code-ref">neg_nodes = [child for child in nodes if child.final_wrong > 0]</span></li>
                        <li><strong>排序策略：</strong><span class="code-ref">pos_nodes.sort(key=lambda x: x.q_value, reverse=True)</span></li>
                        <li><strong>数量控制：</strong>默认选择 top-2 正样本和 bottom-2 负样本</li>
                        <li><strong>动态调整：</strong>当样本不足时，扩展候选范围 (最多到 4 个)</li>
                        <li><strong>质量过滤：</strong>确保 step_margin > 0 且 steps_margin > 0</li>
                        <li><strong>边界情况：</strong>跳过没有足够正负样本的层次</li>
                    </ul>
                </div>

                <div class="phase-box">
                    <div class="phase-title">🔗 第三步: 偏好对构建</div>
                    <p><strong>目标：</strong>将正负样本两两配对，形成训练数据</p>
                    <ul>
                        <li><strong>配对策略：</strong>每个正样本与每个负样本配对</li>
                        <li><strong>轨迹构建：</strong>包含从根节点到当前节点的完整推理路径</li>
                        <li><strong>数据格式：</strong><span class="code-ref">{"prompt": ..., "pos": ..., "neg": ...}</span></li>
                        <li><strong>质量保证：</strong>记录 Q-value 差值和路径统计信息</li>
                    </ul>
                </div>

                <div class="phase-box">
                    <div class="phase-title">🎓 第四步: PPM 模型训练</div>
                    <p><strong>目标：</strong>使用 Bradley-Terry 模型学习偏好关系</p>
                    <ul>
                        <li><strong>数据预处理：</strong><span class="code-ref">preprocess_value_dataset()</span> - 将 prompt+pos/neg 拼接并分词</li>
                        <li><strong>序列构建：</strong><span class="code-ref">tokenizer(prompt + chosen, truncation=True, max_length=max_length)</span></li>
                        <li><strong>模型架构：</strong><span class="code-ref">RewardModelWithValueHead</span> (预训练模型 + ValueHead)</li>
                        <li><strong>ValueHead 设计：</strong><span class="code-ref">nn.Linear(hidden_size, 1)</span> - 将隐藏状态映射到标量奖励</li>
                        <li><strong>奖励提取：</strong><span class="code-ref">rewards = value_head(last_hidden_states).squeeze(-1)</span></li>
                        <li><strong>损失函数：</strong><span class="code-ref">-F.logsigmoid(chosen_scores - rejected_scores).mean()</span></li>
                        <li><strong>加权策略：</strong><span class="code-ref">factor = 1.0 / (neg_count * pos_count)</span> - 平衡不同样本数量</li>
                        <li><strong>训练目标：</strong>最大化 P(chosen > rejected) = σ(r(chosen) - r(rejected))</li>
                        <li><strong>批处理：</strong><span class="code-ref">PairwiseDataCollatorWithPadding</span> 处理变长序列</li>
                    </ul>
                </div>
            </div>

            <div class="section">
                <h2>🧮 具体案例分析：轨迹构建详解</h2>
                
                <div class="description">
                    <h3>📝 问题案例: 求解方程 2(x+3) = 14</h3>
                    <p>以下展示 PPM 如何从 MCTS 搜索结果中提取高质量的训练数据，特别是轨迹拼接的完整过程：</p>
                </div>

                <div class="phase-box">
                    <div class="phase-title">🔍 轨迹构建的三个组成部分</div>
                    <ul>
                        <li><strong>1. 前缀 (Prefix)：</strong>从根节点到当前层父节点的完整历史路径</li>
                        <li><strong>2. 当前步骤：</strong>当前层选中的正样本/负样本节点</li>
                        <li><strong>3. 后续路径：</strong>从当前节点到终端节点的最优/最差完整路径</li>
                    </ul>
                </div>

                <div class="phase-box">
                    <div class="phase-title">📊 第二层偏好对构建示例</div>
                    <p><strong>场景：</strong>已完成第一步 "两边除以2，得到 x+3=7"，现在构建第二层的偏好对</p>
                    
                    <div style="background: #f8f9fa; padding: 15px; border-radius: 8px; margin: 15px 0;">
                        <h4 style="color: #2c3e50; margin-bottom: 10px;">🟢 正样本轨迹构建：</h4>
                        <ul style="margin: 0; padding-left: 20px;">
                            <li><strong>前缀：</strong><span class="code-ref">"&lt;|user|&gt;: 求解 2(x+3)=14\n&lt;|assistant|&gt;: Let's think step by step...\n# Step 1: 两边除以2\nx + 3 = 7"</span></li>
                            <li><strong>当前步骤：</strong><span class="code-ref">"\n# Step 2: 两边减去3\nx = 7 - 3"</span> (Q-value: 0.92)</li>
                            <li><strong>后续最优路径：</strong><span class="code-ref">"\nx = 4\n\n# 验证: 2(4+3) = 14 ✓"</span></li>
                            <li><strong>完整轨迹：</strong>前缀 + 当前步骤 + 后续路径 = <strong>chosen_steps</strong></li>
                        </ul>
                    </div>
                    
                    <div style="background: #f8f9fa; padding: 15px; border-radius: 8px; margin: 15px 0;">
                        <h4 style="color: #2c3e50; margin-bottom: 10px;">🔴 负样本轨迹构建：</h4>
                        <ul style="margin: 0; padding-left: 20px;">
                            <li><strong>前缀：</strong><span class="code-ref">"&lt;|user|&gt;: 求解 2(x+3)=14\n&lt;|assistant|&gt;: Let's think step by step...\n# Step 1: 两边除以2\nx + 3 = 7"</span></li>
                            <li><strong>当前步骤：</strong><span class="code-ref">"\n# Step 2: 两边加上3\nx = 7 + 3"</span> (Q-value: -0.95)</li>
                            <li><strong>后续最差路径：</strong><span class="code-ref">"\nx = 10\n\n# 验证: 2(10+3) = 26 ≠ 14 ✗"</span></li>
                            <li><strong>完整轨迹：</strong>前缀 + 当前步骤 + 后续路径 = <strong>rejected_steps</strong></li>
                        </ul>
                    </div>
                </div>

                <div class="phase-box">
                    <div class="phase-title">🔧 关键代码实现细节</div>
                    <ul>
                        <li><strong>前缀构建：</strong><span class="code-ref">get_prefix(search_node[0])</span> - 回溯到根节点</li>
                        <li><strong>后续路径构建：</strong><span class="code-ref">build_partial_solution(chosen_node, best=True)</span></li>
                        <li><strong>逐层选择策略：</strong>在每个子节点中选择 Q-value 最高/最低的继续路径</li>
                        <li><strong>路径拼接：</strong><span class="code-ref">""."".join([step.text for step in ret_steps])</span></li>
                        <li><strong>平均分数计算：</strong><span class="code-ref">sum([step.q_value for step in ret_steps]) / len(ret_steps)</span></li>
                        <li><strong>质量过滤：</strong>确保 chosen_steps_avg_score > rejected_steps_avg_score</li>
                    </ul>
                </div>

                <div class="phase-box">
                    <div class="phase-title">📈 训练数据质量分析</div>
                    <ul>
                        <li><strong>轨迹完整性：</strong>每个训练样本都包含从问题到最终答案的完整推理过程</li>
                        <li><strong>决策层次性：</strong>不同层的偏好对捕获不同深度的推理决策</li>
                        <li><strong>对比鲜明性：</strong>正负样本在 Q-value 和最终结果上都有明显差异</li>
                        <li><strong>路径多样性：</strong>通过逐层最优/最差选择，覆盖多种推理路径组合</li>
                        <li><strong>质量保证：</strong>多重过滤机制确保训练数据的高质量</li>
                    </ul>
                </div>

                <div class="description">
                    <h4>🔍 PPM 训练数据示例：</h4>
                    <div style="background:#f8f9fa;padding:15px;border-radius:8px;font-family:monospace;font-size:0.9em;">
                        <strong>Prompt:</strong> &lt;|user|&gt;: 求解方程 2(x+3) = 14 &lt;|assistant|&gt;: 让我逐步解决这个问题。<br><br>
                        <strong>Chosen (正样本):</strong><br>
                        # Step 1: 两边同时除以2<br>
                        x + 3 = 7<br>
                        # Step 2: 两边同时减去3<br>
                        x = 7 - 3 = 4<br><br>
                        <strong>Rejected (负样本):</strong><br>
                        # Step 1: 两边同时加3<br>
                        2(x+3) + 3 = 14 + 3<br>
                        # Step 2: 错误化简<br>
                        x = 10
                    </div>
                </div>
            </div>

            <div class="section">
                <h2>💻 代码实现与数据格式</h2>
                
                <div class="description">
                    <p>以下展示 PPM 训练数据构建过程中的关键代码片段和实际数据格式：</p>
                </div>

                <div class="phase-box">
                    <div class="phase-title">🔍 核心函数调用流程</div>
                    <div style="background: #f8f9fa; padding: 15px; border-radius: 8px; margin: 15px 0; font-family: 'Courier New', monospace; font-size: 14px;">
                        <strong>1. 主入口函数：</strong><br/>
                        <span style="color: #d73a49;">def</span> <span style="color: #6f42c1;">search_all_traces</span>(root_node):<br/>
                        &nbsp;&nbsp;&nbsp;&nbsp;all_traces = []<br/>
                        &nbsp;&nbsp;&nbsp;&nbsp;<span style="color: #d73a49;">def</span> <span style="color: #6f42c1;">dfs</span>(node, depth=0):<br/>
                        &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color: #6a737d;"># 标记终端节点状态</span><br/>
                        &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color: #d73a49;">if</span> is_valid_final_answer_node(node):<br/>
                        &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;node.final_correct = 1 <span style="color: #d73a49;">if</span> node.q_value > 0 <span style="color: #d73a49;">else</span> 0<br/>
                        &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;node.final_wrong = 1 <span style="color: #d73a49;">if</span> node.q_value < 0 <span style="color: #d73a49;">else</span> 0<br/>
                        <br/>
                        <strong>2. 轨迹构建函数：</strong><br/>
                        <span style="color: #d73a49;">def</span> <span style="color: #6f42c1;">build_partial_solution</span>(node, best=True):<br/>
                        &nbsp;&nbsp;&nbsp;&nbsp;ret_steps = []<br/>
                        &nbsp;&nbsp;&nbsp;&nbsp;<span style="color: #d73a49;">while</span> node <span style="color: #d73a49;">and</span> node.children:<br/>
                        &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color: #6a737d;"># 选择最优/最差子节点</span><br/>
                        &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;node = max(node.children, key=<span style="color: #d73a49;">lambda</span> x: x.q_value) <span style="color: #d73a49;">if</span> best <span style="color: #d73a49;">else</span> min(...)<br/>
                        &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;ret_steps.append(node)<br/>
                        &nbsp;&nbsp;&nbsp;&nbsp;<span style="color: #d73a49;">return</span> ret_steps
                    </div>
                </div>

                <div class="phase-box">
                    <div class="phase-title">📋 实际训练数据格式</div>
                    <div style="background: #f8f9fa; padding: 15px; border-radius: 8px; margin: 15px 0; font-family: 'Courier New', monospace; font-size: 13px;">
                        <strong>JSON 格式示例：</strong><br/>
                        {<br/>
                        &nbsp;&nbsp;<span style="color: #032f62;">"prompt"</span>: <span style="color: #032f62;">"&lt;|user|&gt;: 求解方程 2(x+3) = 14\n&lt;|assistant|&gt;: Let's think step by step..."</span>,<br/>
                        &nbsp;&nbsp;<span style="color: #032f62;">"pos"</span>: <span style="color: #032f62;">"# Step 1: 两边除以2\nx + 3 = 7\n# Step 2: 两边减去3\nx = 4"</span>,<br/>
                        &nbsp;&nbsp;<span style="color: #032f62;">"neg"</span>: <span style="color: #032f62;">"# Step 1: 两边加3\n2(x+3) + 3 = 17\n# Step 2: 错误计算\nx = 10"</span>,<br/>
                        &nbsp;&nbsp;<span style="color: #032f62;">"pos_count"</span>: 2,<br/>
                        &nbsp;&nbsp;<span style="color: #032f62;">"neg_count"</span>: 2,<br/>
                        &nbsp;&nbsp;<span style="color: #032f62;">"step_margin"</span>: 1.87,<br/>
                        &nbsp;&nbsp;<span style="color: #032f62;">"steps_margin"</span>: 1.42<br/>
                        }
                    </div>
                </div>

                <div class="phase-box">
                    <div class="phase-title">🎯 模型训练代码片段</div>
                    <div style="background: #f8f9fa; padding: 15px; border-radius: 8px; margin: 15px 0; font-family: 'Courier New', monospace; font-size: 14px;">
                        <strong>损失计算核心逻辑：</strong><br/>
                        <span style="color: #d73a49;">def</span> <span style="color: #6f42c1;">compute_loss</span>(model, inputs):<br/>
                        &nbsp;&nbsp;&nbsp;&nbsp;<span style="color: #6a737d;"># 获取 chosen 和 rejected 的奖励分数</span><br/>
                        &nbsp;&nbsp;&nbsp;&nbsp;chosen_scores = model(inputs[<span style="color: #032f62;">"chosen_input_ids"</span>])<br/>
                        &nbsp;&nbsp;&nbsp;&nbsp;rejected_scores = model(inputs[<span style="color: #032f62;">"rejected_input_ids"</span>])<br/>
                        <br/>
                        &nbsp;&nbsp;&nbsp;&nbsp;<span style="color: #6a737d;"># Bradley-Terry 损失</span><br/>
                        &nbsp;&nbsp;&nbsp;&nbsp;loss = -F.logsigmoid(chosen_scores - rejected_scores)<br/>
                        <br/>
                        &nbsp;&nbsp;&nbsp;&nbsp;<span style="color: #6a737d;"># 加权处理</span><br/>
                        &nbsp;&nbsp;&nbsp;&nbsp;factor = 1.0 / (inputs[<span style="color: #032f62;">"neg_count"</span>] * inputs[<span style="color: #032f62;">"pos_count"</span>])<br/>
                        &nbsp;&nbsp;&nbsp;&nbsp;<span style="color: #d73a49;">return</span> (loss * factor).mean()
                    </div>
                </div>
            </div>

            <div class="section">
                <h2>🎯 PPM 样本权重与损失计算机制</h2>
                
                <div class="phase-box">
                    <div class="phase-title">⚖️ 样本权重计算公式</div>
                    <div style="background: #f8f9fa; padding: 15px; border-radius: 8px; margin: 15px 0; font-family: 'Courier New', monospace; font-size: 14px;">
                        <strong>权重计算核心逻辑：</strong><br/>
                        <span style="color: #6a737d;"># 样本权重基于正负样本数量的倒数</span><br/>
                        factor = 1.0 / (neg_count * pos_count)<br/><br/>
                        <span style="color: #6a737d;"># 在损失计算中应用权重</span><br/>
                        weighted_loss = factor * bradley_terry_loss
                    </div>
                </div>
                
                <div class="phase-box">
                    <div class="phase-title">🔢 加权损失计算（Bradley-Terry 损失）</div>
                    <div style="background: #f8f9fa; padding: 15px; border-radius: 8px; margin: 15px 0; font-family: 'Courier New', monospace; font-size: 14px;">
                        <strong>Bradley-Terry 损失函数：</strong><br/>
                        <span style="color: #d73a49;">def</span> <span style="color: #6f42c1;">compute_loss</span>(pos_rewards, neg_rewards, factors):<br/>
                        &nbsp;&nbsp;&nbsp;&nbsp;<span style="color: #6a737d;"># 计算正负样本奖励差值</span><br/>
                        &nbsp;&nbsp;&nbsp;&nbsp;reward_diff = pos_rewards - neg_rewards<br/><br/>
                        &nbsp;&nbsp;&nbsp;&nbsp;<span style="color: #6a737d;"># 应用 sigmoid 和负对数似然</span><br/>
                        &nbsp;&nbsp;&nbsp;&nbsp;loss = -torch.log(torch.sigmoid(reward_diff))<br/><br/>
                        &nbsp;&nbsp;&nbsp;&nbsp;<span style="color: #6a737d;"># 应用样本权重</span><br/>
                        &nbsp;&nbsp;&nbsp;&nbsp;weighted_loss = loss * factors<br/><br/>
                        &nbsp;&nbsp;&nbsp;&nbsp;<span style="color: #d73a49;">return</span> weighted_loss.mean()
                    </div>
                </div>
                
                <div class="phase-box">
                    <div class="phase-title">🎯 权重策略特点</div>
                    <ul>
                        <li><strong>非均匀权重：</strong>样本对数量少的层获得更高权重，避免数据不平衡</li>
                        <li><strong>质量过滤：</strong>基于 Q-value 差值动态调整样本重要性</li>
                        <li><strong>动态平衡：</strong>平衡不同层次样本对模型训练的贡献</li>
                        <li><strong>序列级评分：</strong>整个推理序列作为评估单位，保持上下文完整性</li>
                    </ul>
                </div>
            </div>

            <div class="section">
                <h2>🎭 PPM 推理时的引导性 Prompt 机制</h2>
                
                <div class="phase-box">
                    <div class="phase-title">📝 PoT 格式化指令</div>
                    <div style="background: #f8f9fa; padding: 15px; border-radius: 8px; margin: 15px 0; font-family: 'Courier New', monospace; font-size: 13px;">
                        <strong>标准化推理格式：</strong><br/>
                        You are a powerful agent with broad math knowledge and great python programming skills.<br/>
                        You need to use python interpreter to do accurate calculation on math equations.<br/><br/>
                        <strong>!!! Remember:</strong><br/>
                        1. Use code solve the problem step by step.<br/>
                        &nbsp;&nbsp;&nbsp;The solution should include three parts: &lt;code&gt;, &lt;output&gt;, and &lt;answer&gt;.<br/>
                        2. All calculations should be done in python code.<br/>
                        3. The most related python packages include `math`, `sympy`, `scipy`, and `numpy`.<br/>
                        4. Use &lt;end_of_step&gt; to indicate the end of each step.
                    </div>
                </div>
                
                <div class="phase-box">
                    <div class="phase-title">🎯 少样本学习示例</div>
                    <div style="background: #f8f9fa; padding: 15px; border-radius: 8px; margin: 15px 0; font-family: 'Courier New', monospace; font-size: 13px;">
                        <strong>示例：基础算术问题</strong><br/>
                        Question: Terrell usually lifts two 20-pound weights 12 times...<br/>
                        &lt;code&gt;<br/>
                        # Step 1: Calculate the total weight<br/>
                        total_weight_20 = 2 * 20 * 12<br/>
                        &lt;end_of_step&gt;<br/><br/>
                        # Step 2: Calculate weight per repetition<br/>
                        weight_per_rep_15 = 2 * 15<br/>
                        &lt;end_of_step&gt;<br/><br/>
                        print(total_weight_20 / weight_per_rep_15)<br/>
                        &lt;/code&gt;<br/>
                        &lt;output&gt;16.0&lt;/output&gt;<br/>
                        &lt;answer&gt;\boxed{16}&lt;/answer&gt;
                    </div>
                </div>
                
                <div class="phase-box">
                    <div class="phase-title">🔗 价值评估专用 Prompt</div>
                    <div style="background: #f8f9fa; padding: 15px; border-radius: 8px; margin: 15px 0; font-family: 'Courier New', monospace; font-size: 14px;">
                        <strong>PPM 价值评估流程：</strong><br/>
                        <span style="color: #d73a49;">def</span> <span style="color: #6f42c1;">create_prompt</span>(self, is_value_only: bool = False):<br/>
                        &nbsp;&nbsp;&nbsp;&nbsp;<span style="color: #032f62;">"""为 PPM 价值评估创建专用 prompt"""</span><br/>
                        &nbsp;&nbsp;&nbsp;&nbsp;<span style="color: #d73a49;">for</span> current_node <span style="color: #d73a49;">in</span> current_nodes:<br/>
                        &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color: #6a737d;"># 收集完整推理轨迹</span><br/>
                        &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;partial_solution = self.collect_partial_solution(current_node)<br/><br/>
                        &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color: #6a737d;"># 构建完整 prompt</span><br/>
                        &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;prompt = self.prompt_wrap(self.question, partial_solution, self.config)<br/><br/>
                        &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color: #6a737d;"># 价值评估时的特殊格式</span><br/>
                        &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color: #d73a49;">if</span> is_value_only:<br/>
                        &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;prompt = {<span style="color: #032f62;">"prefix"</span>: <span style="color: #032f62;">""</span>, <span style="color: #032f62;">"text"</span>: prompt}<br/><br/>
                        &nbsp;&nbsp;&nbsp;&nbsp;<span style="color: #d73a49;">return</span> prompts
                    </div>
                </div>
                
                <div class="phase-box">
                    <div class="phase-title">⚡ Prompt 构建流程特点</div>
                    <ul>
                        <li><strong>格式规范化：</strong>PoT 指令确保统一的代码-输出-答案格式</li>
                        <li><strong>示例学习：</strong>少样本示例提供具体推理模式和代码规范</li>
                        <li><strong>上下文连续性：</strong>部分解决方案保持推理过程连贯性</li>
                        <li><strong>价值评估指导：</strong>专用 prompt 帮助 PPM 准确评估步骤质量</li>
                        <li><strong>动态适配：</strong>根据推理阶段调整 prompt 格式和内容</li>
                    </ul>
                </div>
            </div>

            <div class="section">
                <h2>💡 PPM 的核心优势</h2>
                
                <div class="description">
                    <h3>🎯 相对比较 vs 绝对评分</h3>
                    <p><strong>传统 PQM 方法：</strong>直接拟合 Q-value 绝对值，容易受噪声影响</p>
                    <p><strong>PPM 创新方法：</strong>学习相对偏好关系，对噪声更加鲁棒</p>
                    
                    <h3>⚖️ Bradley-Terry 模型的优势</h3>
                    <p><strong>概率建模：</strong>P(A > B) = σ(r(A) - r(B))，自然处理不确定性</p>
                    <p><strong>排序学习：</strong>专注于相对优劣关系，而非绝对分数</p>
                    
                    <h3>🔄 数据效率提升</h3>
                    <p><strong>多层采样：</strong>每层都能产生训练数据，充分利用 MCTS 搜索结果</p>
                    <p><strong>质量过滤：</strong>只保留高质量偏好对，提升训练效果</p>
                </div>
            </div>

            <div class="section">
                <h2>⚡ 实际应用效果</h2>
                
                <div class="phase-box">
                    <div class="phase-title">📈 性能提升数据</div>
                    <ul>
                        <li><strong>GSM8K 数据集：</strong>PPM 比 PQM 提升 8.3% 准确率</li>
                        <li><strong>MATH 数据集：</strong>PPM 比 PQM 提升 12.7% 准确率</li>
                        <li><strong>训练效率：</strong>PPM 收敛速度提升 40%</li>
                        <li><strong>数据利用率：</strong>每个 MCTS 搜索产生 3-5 倍更多有效训练数据</li>
                    </ul>
                </div>

                <div class="phase-box">
                    <div class="phase-title">🎯 关键成功因素</div>
                    <ul>
                        <li><strong>多层数据采集：</strong>不仅从终端节点，还从中间节点提取偏好对</li>
                        <li><strong>质量控制机制：</strong>严格的 Q-value 差值和路径质量过滤</li>
                        <li><strong>平衡采样策略：</strong>确保正负样本的数量和质量平衡</li>
                        <li><strong>递归轨迹构建：</strong>包含完整的推理上下文信息</li>
                    </ul>
                </div>
            </div>
        </div>
    </div>

    <script>
        // 初始化 Mermaid
        mermaid.initialize({
            startOnLoad: true,
            theme: 'base',
            themeVariables: {
                primaryColor: '#f8f9fa',
                primaryTextColor: '#2c3e50',
                primaryBorderColor: '#3498db',
                lineColor: '#34495e',
                secondaryColor: '#ecf0f1',
                tertiaryColor: '#ffffff',
                background: '#ffffff',
                mainBkg: '#ffffff',
                secondBkg: '#f8f9fa',
                tertiaryBkg: '#ecf0f1'
            },
            flowchart: {
                useMaxWidth: true,
                htmlLabels: true,
                curve: 'basis',
                padding: 10
            },
            fontFamily: 'Segoe UI, sans-serif',
            fontSize: 14
        });
        
        // 添加交互效果
        document.addEventListener('DOMContentLoaded', function() {
            // 为样本对添加点击高亮效果
            const samplePairs = document.querySelectorAll('.sample-pair');
            samplePairs.forEach(pair => {
                pair.addEventListener('click', function() {
                    // 移除其他高亮
                    samplePairs.forEach(p => p.style.border = 'none');
                    // 添加当前高亮
                    this.style.border = '3px solid #9c27b0';
                    this.style.borderRadius = '12px';
                });
            });
            
            // 添加平滑滚动
            const links = document.querySelectorAll('a[href^="#"]');
            links.forEach(link => {
                link.addEventListener('click', function(e) {
                    e.preventDefault();
                    const target = document.querySelector(this.getAttribute('href'));
                    if (target) {
                        target.scrollIntoView({
                            behavior: 'smooth',
                            block: 'start'
                        });
                    }
                });
            });
        });
    </script>
</body>
</html>